#!/usr/bin/python3.5
from urllib.request import urlopen
from urllib.request import Request
from urllib.error import URLError
from urllib.error import HTTPError
from bs4 import BeautifulSoup
import module

tuplePerProcess = 5

def main():
    index = module.getStartIndex()  # get the last index of previous execution as current start index

    # network request parameters
    interpreter = ["html.parser", "lxml", "xml", "html5lib"]
    user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0"
    headers = {"User-Agent": user_agent}
    indexUrl = "http://www.theplantlist.org/tpl1.1/record/"
    times = 0
    while True:
        #if (times / tuplePerProcess) == 1:  # jump out the loop to accomplish the program per 10 times, so every loop self-increase 1
        #   break
        # open the webpage file and parse the content
        try:
            pageUrl = "kew-" + str(index)
            entireUrl = indexUrl + pageUrl
            webpage = Request(entireUrl, None, headers)
            html = urlopen(webpage)
        except URLError as e1:
            print("(%d)url open ERROR!,reason is %s" % (index, e1.reason))
            module.recordLastIndex(index)   # firstly record the current index before self-increase
            times += 1
            index += 1
            continue
        except HTTPError as e2:
            print("(%d)url open ERROR!,error code is %d, reason is %s" % (index, e2.code, e2.reason))
            module.recordLastIndex(index)   # firstly record the current index before self-increase
            times += 1
            index += 1
            continue

        bsObj = BeautifulSoup(html, interpreter[1]) # create the bsObj object

        # crawl the info of every tuple
        try:
            sciName = bsObj.find("span", {"class": "name"}).get_text()
            status = bsObj.find("span", {"class": "subtitle"}).findAll("a")[0].get_text().strip()
            synonym = ""
            if status == "synonym":
                synonym = bsObj.find("span", {"class": "subtitle"}).findAll("a")[1].get_text().strip()
                #synonym = bsObj.find("span", {"class": "subtitle"}).find("span",{"class":"name"}).get_text().strip()
                module.writeLog(index, sciName, status, synonym)
                module.recordLastIndex(index)  # firstly record the current index before self-increase
                index += 1
                formatStr = "<%d> (SCINAM: %s), (STATUS: %s), (SYNONYM: %s)" % (index, sciName, status, synonym)
                print(formatStr)
            else:
                module.writeLog(index, sciName, status, synonym)
                module.recordLastIndex(index)  # firstly record the current index before self-increase
                index += 1
                formatStr = "<%d> (SCINAM: %s), (STATUS: %s), (SYNONYM: %s)" % (index, sciName, status, synonym)
                print(formatStr)
            times += 1
        except:
            print(str(index) + "< ERROR >")
            module.recordLastIndex(index)   # firstly record the current index before self-increase
            index += 1
            times += 1
            continue

if __name__ == '__main__':
    main()